{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    " \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.externals import joblib \n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import RidgeCV\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib.pyplot as plt\n",
    "import xgboost as xgb\n",
    "from lightgbm import LGBMRegressor\n",
    "import math\n",
    "from catboost import Pool, CatBoostRegressor\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据预处理\n",
    "def clean_data(train_data_X):    \n",
    "    for j in range(len(np.array(train_data_X.columns))):\n",
    "        if train_data_X.columns[j] != \"ID\":\n",
    "            print(\"正在处理第%d列数据\"%j)\n",
    "            array = train_data_X[train_data_X.columns[j]]\n",
    "            num = 0\n",
    "            len_ = len(train_data_X[train_data_X.columns[j]])\n",
    "            for i in range(len_):   \n",
    "                std_ = array.std()\n",
    "                min_ = array.min()\n",
    "                max_ = array.max()\n",
    "                mean_ = array.mean()\n",
    "                num = array[i]\n",
    "                std_max =np.float(mean_+3*std_)\n",
    "                std_min = np.float(mean_-3*std_)\n",
    "                if (array[0]>std_max) | (array[0]<std_min):\n",
    "    #                 mean_del = mean_-(array[0]-mean_)/len_\n",
    "    #                 array[0] = mean_del\n",
    "                    array[0] = mean_\n",
    "                if (array[len_-1]>std_max) | (array[len_-1]<std_min):\n",
    "                    array[len_-1] = mean_\n",
    "                if i>0 and i<(len_-1):\n",
    "                    if (array[i]>std_max) | (array[i]<std_min):\n",
    "                        if (array[i-1]<std_max) and (array[i-1]>std_min) and (array[i+1]<std_max) and (array[i+1]>std_min):\n",
    "                            array[i]= np.float(array[i-1]+array[i+1])/2\n",
    "                        elif ((array[i-1]<std_max) and (array[i-1]>std_min))| (array[i+1]>std_max) | (array[i+1]<std_min): \n",
    "                            array[i] = array[i-1]\n",
    "                        elif (array[i-1]<std_max) | (array[i-1]>std_min)|((array[i+1]<std_max) and (array[i+1]>std_min)):\n",
    "                            array[i] = array[i+1]\n",
    "            train_data_X[train_data_X.columns[j]] = array\n",
    "        else:\n",
    "            pass\n",
    "            \n",
    "    return train_data_X\n",
    "def drop_all_outlier(df):\n",
    "    df.drop_duplicates(df.columns.drop('ID'), keep='first', inplace=True)\n",
    "    df.drop(df[(df.电压A > 800) | (df.电压A < 500)].index,inplace=True)\n",
    "    df.drop(df[(df.电压B > 800) | (df.电压B < 500)].index,inplace=True)\n",
    "    df.drop(df[(df.电压C > 800) | (df.电压C < 500)].index,inplace=True)\n",
    "    df.drop(df[(df.现场温度 > 30) | (df.现场温度 < -30)].index,inplace=True)\n",
    "    df.drop(df[(df.转换效率A > 100)].index,inplace=True)\n",
    "    df.drop(df[(df.转换效率B > 100)].index,inplace=True)\n",
    "    df.drop(df[(df.转换效率C > 100)].index,inplace=True)\n",
    "    df.drop(df[(df.风向 > 360)].index,inplace=True)\n",
    "    df.drop(df[(df.风速 > 20)].index,inplace=True)\n",
    "    return df\n",
    "# 生成数据\n",
    "def generate_train_data(train_data, test_data, poly=False, select=False):\n",
    "    y = train_data['发电量']\n",
    "    X = train_data.drop(['发电量','ID'], axis=1)\n",
    "    sub_data = test_data.drop(['ID'], axis=1)\n",
    "    \n",
    "    polynm = None\n",
    "    if poly:\n",
    "        from sklearn.preprocessing import PolynomialFeatures\n",
    "        polynm = PolynomialFeatures(degree=2, interaction_only=True)\n",
    "        X = polynm.fit_transform(X)\n",
    "        sub_data = polynm.transform(sub_data)\n",
    "        \n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n",
    "    \n",
    "    sm = None\n",
    "    if select:\n",
    "        from sklearn.feature_selection import SelectFromModel\n",
    "        sm = SelectFromModel(GradientBoostingRegressor(random_state=2))\n",
    "        X_train = sm.fit_transform(X_train, y_train)\n",
    "        X_test = sm.transform(X_test)\n",
    "        sub_data = sm.transform(sub_data)\n",
    "        \n",
    "    return X_train, X_test, y_train, y_test, sub_data, sm, polynm\n",
    "\n",
    "def cal_score(mse):\n",
    "    if isinstance(mse, float):\n",
    "        return 1 / (1 + math.sqrt(mse))\n",
    "    else:\n",
    "        return np.divide(1, 1 + np.sqrt(mse))\n",
    "#  定义交叉验证函数  \n",
    "def cross_validation_test(models, train_X_data, train_y_data, cv=5):\n",
    "    model_name, mse_avg, score_avg = [], [], []\n",
    "    for i, model in enumerate(models):\n",
    "        print(i + 1,'- Model:', str(model).split('(')[0])\n",
    "        model_name.append(str(i + 1) + '.' + str(model).split('(')[0])\n",
    "        nmse = cross_val_score(model, train_X_data[i], train_y_data[i], cv=cv, scoring='neg_mean_squared_error')\n",
    "        avg_mse = np.average(-nmse)\n",
    "        scores = cal_score(-nmse)\n",
    "        avg_score = np.average(scores)\n",
    "        mse_avg.append(avg_mse)\n",
    "        score_avg.append(avg_score)\n",
    "        print('MSE:', -nmse)\n",
    "        print('Score:', scores)\n",
    "        print('Average XGB - MSE:', avg_mse, ' - Score:', avg_score, '\\n')\n",
    "    res = pd.DataFrame()\n",
    "    res['Model'] = model_name\n",
    "    res['Avg MSE'] = mse_avg\n",
    "    res['Avg Score'] = score_avg\n",
    "    return res\n",
    "\n",
    "def add_avg(df):\n",
    "    array = np.array(df[\"平均功率\"])\n",
    "    newarray=[]\n",
    "    num = 0\n",
    "    for i in np.arange(len(array)):\n",
    "        for j in np.arange(10):\n",
    "            sum_1=0\n",
    "            for m in np.arange(j):\n",
    "                sum_1 += array[m]\n",
    "            if i<10:\n",
    "                num = sum_1/(j+1)\n",
    "            else:\n",
    "                num = (array[i-1]+array[i-2]+array[i-3]+array[i-4]+array[i-5]+array[i-6]+array[i-7]+array[i-8]+array[i-9])/9\n",
    "        newarray.append(num)\n",
    "    df[\"old平均功率\"] = newarray\n",
    "    return df\n",
    "def add_avgs(df,alpha,str_1,str_2):\n",
    "    array = np.array(df[str_1])\n",
    "    for i in np.arange(len(array)):\n",
    "        if i==0:\n",
    "            array[i]=alpha*0+(1-alpha)*array[i]\n",
    "        else:\n",
    "            array[i]=alpha*array[i-1]+(1-alpha)*array[i]\n",
    "    df[str_2] = array\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(1,2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('../data/public.train.csv')\n",
    "test_data = pd.read_csv('../data/public.test.csv')\n",
    "\n",
    "df_result = pd.DataFrame()\n",
    "df_result['ID'] = list(test_data['ID'])\n",
    "special_missing_ID = test_data[test_data[(test_data == 0) | (test_data == 0.)].count(axis=1) > 13]['ID']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 异常值处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=True'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass sort=False\n",
      "\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "cleaned_train_data = train_data.copy()\n",
    "cleaned_train_data = drop_all_outlier(cleaned_train_data)\n",
    "\n",
    "cleaned_sub_data = test_data.copy()\n",
    "cleaned_sub_data = drop_all_outlier(cleaned_sub_data)\n",
    "cleaned_sub_data_ID = cleaned_sub_data['ID']\n",
    "\n",
    "# all_data = pd.concat([train_data, test_data], axis=0).sort_values(by='ID').reset_index().drop(['index'], axis=1)\n",
    "# bad_feature = ['ID', '功率A', '功率B', '功率C', '平均功率', '现场温度', '电压A', '电压B', '电压C', '电流B', '电流C', '转换效率', '转换效率A', '转换效率B', '转换效率C']\n",
    "# bad_index = all_data[bad_feature][\n",
    "#     (all_data[bad_feature] > all_data[bad_feature].mean() + 2 * all_data[bad_feature].std()) | \n",
    "#     (all_data[bad_feature] < all_data[bad_feature].mean() - 2 * all_data[bad_feature].std())\n",
    "# ].dropna(how='all').index\n",
    "\n",
    "all_data  = pd.concat([train_data, test_data], axis=0).sort_values(by='ID').reset_index().drop(['index'], axis=1)\n",
    "bad_feature = ['ID','功率A', '功率B', '功率C', '平均功率', '现场温度', '电压A', '电压B', '电压C', '电流B', '电流C', '转换效率', '转换效率A', '转换效率B', '转换效率C']\n",
    "bad_index1 = all_data[bad_feature][\n",
    "    (all_data[bad_feature] > all_data[bad_feature].mean() + 2 * all_data[bad_feature].std()) | \n",
    "    (all_data[bad_feature] < all_data[bad_feature].mean() - 2 * all_data[bad_feature].std())\n",
    "].dropna(how='all').index\n",
    "bad_index2 = all_data[\n",
    "    ((all_data['电压A']<500)&(all_data['电压A']!=0))|\n",
    "    ((all_data['电压B']<500)&(all_data['电压B']!=0))|\n",
    "    ((all_data['电压C']<500)&(all_data['电压C']!=0))].index\n",
    "bad_index = pd.Int64Index(list(bad_index1)+list(bad_index2))\n",
    "# all_data.loc[np.concatenate([bad_index -1,bad_index,bad_index+1])].sort_values(by='ID', ascending=True)\n",
    "\n",
    "\n",
    "nn_bad_data = all_data.loc[np.concatenate([bad_index - 1, bad_index, bad_index + 1])].sort_values(by='ID', ascending=True).drop_duplicates()\n",
    "bad_data = all_data.loc[bad_index].sort_values(by='ID', ascending=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 上下记录均值替代异常值\n",
    "for idx, line in bad_data.iterrows():\n",
    "    ID = line['ID']\n",
    "    col_index = line[bad_feature][ \n",
    "        (line[bad_feature] > all_data[bad_feature].mean() + 3 * all_data[bad_feature].std())| \n",
    "        (line[bad_feature] < all_data[bad_feature].mean() - 3 * all_data[bad_feature].std())\n",
    "    ].index\n",
    "    index = all_data[all_data['ID'] == ID].index\n",
    "    \n",
    "    before_offset = 1\n",
    "    while (idx + before_offset)in bad_index:\n",
    "        before_offset += 1\n",
    "\n",
    "    after_offset = 1\n",
    "    while (idx + after_offset) in bad_index:\n",
    "        after_offset += 1\n",
    "    \n",
    "    replace_value = (all_data.loc[index - before_offset, col_index].values + all_data.loc[index + after_offset, col_index].values) / 2\n",
    "    all_data.loc[index, col_index] = replace_value[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 拆分数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#拆分数据\n",
    "train_data = all_data.drop(all_data[all_data['ID'].isin(df_result['ID'])].index).reset_index().drop(['index'], axis=1)\n",
    "test_data = all_data[all_data['ID'].isin(df_result['ID'])].drop(['发电量'], axis=1).reset_index().drop(['index'], axis=1)\n",
    "len(train_data), len(test_data)\n",
    "# 去除重复值\n",
    "train_data = train_data.drop_duplicates(train_data.columns.drop('ID'), keep='first')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = add_avg(train_data)\n",
    "test_data = add_avg(test_data)\n",
    "cleaned_train_data = add_avg(cleaned_train_data)\n",
    "cleaned_sub_data = add_avg(cleaned_sub_data)\n",
    "\n",
    "# train_data = add_avg(train_data,0.9,\"平均功率\",\"平均功率_历史数据\")\n",
    "# test_data = add_avg(test_data,0.9,\"平均功率\",\"平均功率_历史数据\")\n",
    "# cleaned_train_data = add_avg(cleaned_train_data,0.9,\"平均功率\",\"平均功率_历史数据\")\n",
    "# cleaned_sub_data = add_avg(cleaned_sub_data,0.9,\"平均功率\",\"平均功率_历史数据\")\n",
    "\n",
    "# train_data = add_avgs(train_data,0.9,\"功率A\",\"功率A_历史数据\")\n",
    "# test_data = add_avgs(test_data,0.9,\"功率A\",\"功率A_历史数据\")\n",
    "# cleaned_train_data = add_avgs(cleaned_train_data,0.9,\"功率A\",\"功率A_历史数据\")\n",
    "# cleaned_sub_data = add_avgs(cleaned_sub_data,0.9,\"功率A\",\"功率A_历史数据\")\n",
    "\n",
    "train_data = add_avgs(train_data,0.9,\"功率B\",\"功率B_历史数据\")\n",
    "test_data = add_avgs(test_data,0.9,\"功率B\",\"功率B_历史数据\")\n",
    "cleaned_train_data = add_avgs(cleaned_train_data,0.9,\"功率B\",\"功率B_历史数据\")\n",
    "cleaned_sub_data = add_avgs(cleaned_sub_data,0.9,\"功率B\",\"功率B_历史数据\")\n",
    "\n",
    "train_data = add_avgs(train_data,0.9,\"功率C\",\"功率C_历史数据\")\n",
    "test_data = add_avgs(test_data,0.9,\"功率C\",\"功率C_历史数据\")\n",
    "cleaned_train_data = add_avgs(cleaned_train_data,0.9,\"功率C\",\"功率C_历史数据\")\n",
    "cleaned_sub_data = add_avgs(cleaned_sub_data,0.9,\"功率C\",\"功率C_历史数据\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pandas import DataFrame\n",
    "from pandas import Series\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import KFold\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.preprocessing import scale\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "\n",
    "# 代码部分\n",
    "# 读取数据，train_x,train_y,test_x就可以直接用\n",
    "\n",
    "train_x = pd.read_csv('../feature/train_x.csv').values # 训练集X\n",
    "train_y = pd.read_csv('../feature/train_y.csv').values.ravel() # 训练集y\n",
    "test_x = pd.read_csv('../feature/test_x.csv').values   # 线上测试集test_x\n",
    "test_ID = pd.read_csv('../feature/test_ID.csv')        # 这个不用管\n",
    "all_ID = pd.read_csv('../feature/all_ID.csv')          # 也不管\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(train_x, train_y, test_size=0.2, random_state=123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train, X_test, y_train, y_test, sub_data, sm, polynm = generate_train_data(train_x, test_x, poly=True, select=True)\n",
    "\n",
    "# # clean_X_train, clean_X_test, clean_y_train, clean_y_test, clean_sub_data, _, _ = generate_train_data(cleaned_train_data, cleaned_sub_data, poly=False, select=False)\n",
    "\n",
    "# clean_X = np.concatenate([clean_X_train, clean_X_test])\n",
    "# clean_y = np.concatenate([clean_y_train, clean_y_test])\n",
    "# clean_X = polynm.transform(clean_X)\n",
    "# clean_X = sm.transform(clean_X)\n",
    "\n",
    "# clean_sub_data = polynm.transform(clean_sub_data)\n",
    "# clean_sub_data = sm.transform(clean_sub_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tree Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_X_train = np.concatenate([X_train, X_test])\n",
    "all_y_train = np.concatenate([y_train, y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 - Model: XGBRegressor\n",
      "MSE: [0.05529833 0.01467334 0.01883639 0.01771896 0.01761158]\n",
      "Score: [0.80961436 0.8919544  0.87931743 0.88252484 0.88283958]\n",
      "Average XGB - MSE: 0.024827718685883536  - Score: 0.8692501202201539 \n",
      "\n",
      "2 - Model: XGBRegressor\n",
      "MSE: [0.05528646 0.01465626 0.01880667 0.01769189 0.01758168]\n",
      "Score: [0.8096309  0.89201051 0.87940121 0.88260408 0.88292743]\n",
      "Average XGB - MSE: 0.02480458925543899  - Score: 0.8693148237567208 \n",
      "\n",
      "3 - Model: XGBRegressor\n",
      "MSE: [0.05525404 0.01463408 0.01878894 0.01766022 0.01756507]\n",
      "Score: [0.8096761  0.89208341 0.8794512  0.88269686 0.88297625]\n",
      "Average XGB - MSE: 0.024780471001577674  - Score: 0.869376763260887 \n",
      "\n",
      "4 - Model: GradientBoostingRegressor\n",
      "MSE: [0.05682425 0.0148558  0.02004622 0.02109739 0.01719202]\n",
      "Score: [0.80750764 0.89135744 0.87597533 0.87317225 0.88408078]\n",
      "Average XGB - MSE: 0.026003137160532774  - Score: 0.866418687861405 \n",
      "\n",
      "5 - Model: GradientBoostingRegressor\n",
      "MSE: [0.0568036  0.01482577 0.02000118 0.02108444 0.01717382]\n",
      "Score: [0.80753589 0.89145538 0.87609746 0.87320624 0.88413505]\n",
      "Average XGB - MSE: 0.025977761569374952  - Score: 0.8664860066969753 \n",
      "\n",
      "6 - Model: GradientBoostingRegressor\n",
      "MSE: [0.05681276 0.0148375  0.02000989 0.02108385 0.01718535]\n",
      "Score: [0.80752336 0.89141714 0.87607381 0.87320779 0.88410068]\n",
      "Average XGB - MSE: 0.02598586937962948  - Score: 0.8664645560851382 \n",
      "\n",
      "7 - Model: RandomForestRegressor\n",
      "MSE: [0.06101081 0.01680279 0.02411108 0.02817972 0.02084135]\n",
      "Score: [0.80192226 0.88524907 0.86559296 0.85626104 0.87384682]\n",
      "Average XGB - MSE: 0.03018914847565654  - Score: 0.8565744305925804 \n",
      "\n",
      "8 - Model: RandomForestRegressor\n",
      "MSE: [0.06100598 0.01690798 0.0241067  0.02787781 0.02083462]\n",
      "Score: [0.80192855 0.88493173 0.86560352 0.85692264 0.87386462]\n",
      "Average XGB - MSE: 0.03014661681841808  - Score: 0.856650210479424 \n",
      "\n",
      "9 - Model: RandomForestRegressor\n",
      "MSE: [0.06080835 0.01689764 0.02409537 0.02736898 0.021112  ]\n",
      "Score: [0.80218612 0.88496287 0.86563087 0.85804819 0.8731339 ]\n",
      "Average XGB - MSE: 0.030056465870736404  - Score: 0.8567923919286793 \n",
      "\n",
      "10 - Model: LGBMRegressor\n",
      "MSE: [0.05562404 0.01531558 0.02215887 0.0170922  0.01828581]\n",
      "Score: [0.80916132 0.88987277 0.87042916 0.88437884 0.88088263]\n",
      "Average XGB - MSE: 0.02569530147252914  - Score: 0.8669449454909792 \n",
      "\n",
      "11 - Model: LGBMRegressor\n",
      "MSE: [0.05619846 0.01571667 0.02352923 0.01730314 0.01853314]\n",
      "Score: [0.80836683 0.88859969 0.86700761 0.88375025 0.88017594]\n",
      "Average XGB - MSE: 0.026256128771381376  - Score: 0.8655800644465834 \n",
      "\n",
      "12 - Model: LGBMRegressor\n",
      "MSE: [0.05622926 0.01579835 0.02236907 0.01624689 0.01860713]\n",
      "Score: [0.80832438 0.88834286 0.86989583 0.88694683 0.87996568]\n",
      "Average XGB - MSE: 0.02585014130886305  - Score: 0.8666951173956832 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>Avg MSE</th>\n",
       "      <th>Avg Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.XGBRegressor</td>\n",
       "      <td>0.024828</td>\n",
       "      <td>0.869250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.XGBRegressor</td>\n",
       "      <td>0.024805</td>\n",
       "      <td>0.869315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.XGBRegressor</td>\n",
       "      <td>0.024780</td>\n",
       "      <td>0.869377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.GradientBoostingRegressor</td>\n",
       "      <td>0.026003</td>\n",
       "      <td>0.866419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.GradientBoostingRegressor</td>\n",
       "      <td>0.025978</td>\n",
       "      <td>0.866486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6.GradientBoostingRegressor</td>\n",
       "      <td>0.025986</td>\n",
       "      <td>0.866465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7.RandomForestRegressor</td>\n",
       "      <td>0.030189</td>\n",
       "      <td>0.856574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8.RandomForestRegressor</td>\n",
       "      <td>0.030147</td>\n",
       "      <td>0.856650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9.RandomForestRegressor</td>\n",
       "      <td>0.030056</td>\n",
       "      <td>0.856792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10.LGBMRegressor</td>\n",
       "      <td>0.025695</td>\n",
       "      <td>0.866945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11.LGBMRegressor</td>\n",
       "      <td>0.026256</td>\n",
       "      <td>0.865580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12.LGBMRegressor</td>\n",
       "      <td>0.025850</td>\n",
       "      <td>0.866695</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Model   Avg MSE  Avg Score\n",
       "0                1.XGBRegressor  0.024828   0.869250\n",
       "1                2.XGBRegressor  0.024805   0.869315\n",
       "2                3.XGBRegressor  0.024780   0.869377\n",
       "3   4.GradientBoostingRegressor  0.026003   0.866419\n",
       "4   5.GradientBoostingRegressor  0.025978   0.866486\n",
       "5   6.GradientBoostingRegressor  0.025986   0.866465\n",
       "6       7.RandomForestRegressor  0.030189   0.856574\n",
       "7       8.RandomForestRegressor  0.030147   0.856650\n",
       "8       9.RandomForestRegressor  0.030056   0.856792\n",
       "9              10.LGBMRegressor  0.025695   0.866945\n",
       "10             11.LGBMRegressor  0.026256   0.865580\n",
       "11             12.LGBMRegressor  0.025850   0.866695"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgbt1 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=789,eta=0.03,scoring='neg_mean_squared_error',n_estimators=351, max_depth=5, max_features='sqrt', random_state=777, n_jobs=12)\n",
    "xgbt2 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=1000,eta=0.03,scoring='neg_mean_squared_error',n_estimators=361, max_depth=5, max_features='sqrt', random_state=999, n_jobs=12)\n",
    "xgbt3 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=1515,eta=0.03,scoring='neg_mean_squared_error',n_estimators=371, max_depth=5, max_features='sqrt', random_state=367, n_jobs=12)\n",
    "# n_estimators=1000  max_depth=5  'sqrt'  GradientBoostingRegressor 最佳参数 ,learning_rate=0.08\n",
    "gbdt1 = GradientBoostingRegressor(n_estimators=1060, max_depth=5, max_features='log2', random_state=789,learning_rate=0.08)\n",
    "gbdt2 = GradientBoostingRegressor(n_estimators=1100, max_depth=5, max_features='log2', random_state=789,learning_rate=0.08)\n",
    "gbdt3 = GradientBoostingRegressor(n_estimators=1090, max_depth=5, max_features='log2', random_state=789,learning_rate=0.08)\n",
    "# n_estimators=700, max_features='auto', random_state=2, n_jobs=8,max_depth=10\n",
    "forest1 = RandomForestRegressor(n_estimators=740, max_features='sqrt', random_state=7, n_jobs=12)\n",
    "forest2 = RandomForestRegressor(n_estimators=730, max_features='sqrt', random_state=9, n_jobs=12)\n",
    "forest3 = RandomForestRegressor(n_estimators=745, max_features='sqrt', random_state=11, n_jobs=12) \n",
    "# n_estimators 850 num_leaves=14,learning_rate=0.1 max_depth=5  seed\n",
    "lgb1 = LGBMRegressor(n_estimators=840, max_depth=4, random_state=789, n_jobs=12,num_leaves=14,learning_rate=0.08,seed=666) \n",
    "lgb2 = LGBMRegressor(n_estimators=845, max_depth=4, random_state=798, n_jobs=12,num_leaves=13,learning_rate=0.1,seed=777)\n",
    "lgb3 = LGBMRegressor(n_estimators=850, max_depth=4, random_state=777, n_jobs=12,num_leaves=10,learning_rate=0.1,seed=999)\n",
    "\n",
    "# xgbt1 = xgb.XGBRegressor(n_estimators=950, max_depth=3, max_features='sqrt', random_state=2, n_jobs=8)\n",
    "# xgbt2 = xgb.XGBRegressor(n_estimators=1000, max_depth=3, max_features='sqrt', random_state=3, n_jobs=8)\n",
    "# xgbt3 = xgb.XGBRegressor(n_estimators=1100, max_depth=3, max_features='sqrt', random_state=4, n_jobs=8)\n",
    "\n",
    "# gbdt1 = GradientBoostingRegressor(n_estimators=500, max_depth=3, max_features='sqrt', random_state=2)\n",
    "# gbdt2 = GradientBoostingRegressor(n_estimators=400, max_depth=3, max_features='sqrt', random_state=3)\n",
    "# gbdt3 = GradientBoostingRegressor(n_estimators=500, max_depth=4, max_features='log2', random_state=4)\n",
    "\n",
    "# forest1 = RandomForestRegressor(n_estimators=300, max_features='sqrt', random_state=2, n_jobs=8)\n",
    "# forest2 = RandomForestRegressor(n_estimators=300, max_features='log2', random_state=3, n_jobs=8)\n",
    "# forest3 = RandomForestRegressor(n_estimators=600, max_features='sqrt', random_state=4, n_jobs=8) \n",
    "\n",
    "# lgb1 = LGBMRegressor(n_estimators=900, max_depth=5, random_state=2, n_jobs=8) \n",
    "# lgb2 = LGBMRegressor(n_estimators=850, max_depth=4, random_state=3, n_jobs=8)\n",
    "# lgb3 = LGBMRegressor(n_estimators=720, max_depth=4, random_state=4, n_jobs=8)\n",
    "\n",
    "cross_validation_test(\n",
    "    models=[    \n",
    "        xgbt1, xgbt2, xgbt3,\n",
    "        gbdt1, gbdt2, gbdt3,\n",
    "        forest1, forest2, forest3,\n",
    "        lgb1, lgb2, lgb3\n",
    "    ],\n",
    "    train_X_data=[\n",
    "        all_X_train, all_X_train, all_X_train, all_X_train,\n",
    "        all_X_train, all_X_train, all_X_train, all_X_train,\n",
    "        all_X_train, all_X_train, all_X_train, all_X_train\n",
    "    ],\n",
    "    train_y_data=[\n",
    "        all_y_train, all_y_train, all_y_train, all_y_train,\n",
    "        all_y_train, all_y_train, all_y_train, all_y_train,\n",
    "        all_y_train, all_y_train, all_y_train, all_y_train\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbt1 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=3000,eta=0.03,scoring='neg_mean_squared_error',n_estimators=351, max_depth=5, max_features='sqrt', random_state=2, n_jobs=12)\n",
    "xgbt2 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=3000,eta=0.03,scoring='neg_mean_squared_error',n_estimators=361, max_depth=5, max_features='sqrt', random_state=3, n_jobs=12)\n",
    "xgbt3 = xgb.XGBRegressor(silent=1,max_leaf_nodes=255,seed=3000,eta=0.03,scoring='neg_mean_squared_error',n_estimators=371, max_depth=5, max_features='sqrt', random_state=4, n_jobs=12)\n",
    "# n_estimators=1000  max_depth=5  'sqrt'  GradientBoostingRegressor 最佳参数 ,learning_rate=0.08\n",
    "gbdt1 = GradientBoostingRegressor(n_estimators=1010, max_depth=5, max_features='log2', random_state=790,learning_rate=0.08)\n",
    "gbdt2 = GradientBoostingRegressor(n_estimators=1000, max_depth=5, max_features='log2', random_state=791,learning_rate=0.08)\n",
    "gbdt3 = GradientBoostingRegressor(n_estimators=1020, max_depth=5, max_features='log2', random_state=789,learning_rate=0.08)\n",
    "# n_estimators=700, max_features='auto', random_state=2, n_jobs=8,max_depth=10\n",
    "forest1 = RandomForestRegressor(n_estimators=720, max_features='sqrt', random_state=7, n_jobs=12)\n",
    "forest2 = RandomForestRegressor(n_estimators=750, max_features='log2', random_state=9, n_jobs=12)\n",
    "forest3 = RandomForestRegressor(n_estimators=800, max_features='sqrt', random_state=11, n_jobs=12) \n",
    "\n",
    "lgb1 = LGBMRegressor(n_estimators=800, max_depth=4, random_state=5, n_jobs=12) \n",
    "lgb2 = LGBMRegressor(n_estimators=850, max_depth=4, random_state=7, n_jobs=12)\n",
    "lgb3 = LGBMRegressor(n_estimators=900, max_depth=4, random_state=9, n_jobs=12)\n",
    "\n",
    "Model\tAvg MSE\tAvg Score\n",
    "0\t1.XGBRegressor\t0.028357\t0.871174\n",
    "1\t2.XGBRegressor\t0.028332\t0.871251\n",
    "2\t3.XGBRegressor\t0.028333\t0.871279\n",
    "3\t4.GradientBoostingRegressor\t0.027985\t0.871623\n",
    "4\t5.GradientBoostingRegressor\t0.027273\t0.872185\n",
    "5\t6.GradientBoostingRegressor\t0.026847\t0.874907\n",
    "6\t7.RandomForestRegressor\t0.030499\t0.865014\n",
    "7\t8.RandomForestRegressor\t0.030333\t0.864953\n",
    "8\t9.RandomForestRegressor\t0.030495\t0.864881\n",
    "9\t10.LGBMRegressor\t0.029263\t0.868385\n",
    "10\t11.LGBMRegressor\t0.029263\t0.868362\n",
    "11\t12.LGBMRegressor\t0.029246\t0.868430"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "xgbt1 = xgb.XGBRegressor(n_estimators=950, max_depth=3, max_features='sqrt', random_state=2, n_jobs=8)\n",
    "xgbt2 = xgb.XGBRegressor(n_estimators=1000, max_depth=3, max_features='sqrt', random_state=3, n_jobs=8)\n",
    "xgbt3 = xgb.XGBRegressor(n_estimators=1100, max_depth=3, max_features='sqrt', random_state=4, n_jobs=8)\n",
    "# n_estimators=1000  max_depth=5  'sqrt'  GradientBoostingRegressor 最佳参数 ,learning_rate=0.08\n",
    "gbdt1 = GradientBoostingRegressor(n_estimators=800, max_depth=4, max_features='log2', random_state=2,learning_rate=0.08)\n",
    "gbdt2 = GradientBoostingRegressor(n_estimators=900, max_depth=4, max_features='log2', random_state=3,learning_rate=0.08)\n",
    "gbdt3 = GradientBoostingRegressor(n_estimators=1000, max_depth=5, max_features='log2', random_state=4,learning_rate=0.08)\n",
    "# n_estimators=700, max_features='auto', random_state=2, n_jobs=8,max_depth=10\n",
    "forest1 = RandomForestRegressor(n_estimators=300, max_features='sqrt', random_state=2, n_jobs=8)\n",
    "forest2 = RandomForestRegressor(n_estimators=400, max_features='log2', random_state=3, n_jobs=8)\n",
    "forest3 = RandomForestRegressor(n_estimators=600, max_features='sqrt', random_state=4, n_jobs=8) \n",
    "\n",
    "lgb1 = LGBMRegressor(n_estimators=900, max_depth=5, random_state=2, n_jobs=8) \n",
    "lgb2 = LGBMRegressor(n_estimators=850, max_depth=4, random_state=3, n_jobs=8)\n",
    "lgb3 = LGBMRegressor(n_estimators=720, max_depth=4, random_state=4, n_jobs=8)\n",
    "\n",
    "Model\tAvg MSE\tAvg Score\n",
    "0\t1.XGBRegressor\t0.029047\t0.869022\n",
    "1\t2.XGBRegressor\t0.029007\t0.869151\n",
    "2\t3.XGBRegressor\t0.028923\t0.869379\n",
    "3\t4.GradientBoostingRegressor\t0.028264\t0.870069\n",
    "4\t5.GradientBoostingRegressor\t0.028909\t0.869134\n",
    "5\t6.GradientBoostingRegressor\t0.027992\t0.872356\n",
    "6\t7.RandomForestRegressor\t0.030729\t0.864344\n",
    "7\t8.RandomForestRegressor\t0.030402\t0.865025\n",
    "8\t9.RandomForestRegressor\t0.030547\t0.864977\n",
    "9\t10.LGBMRegressor\t0.028749\t0.868748\n",
    "10\t11.LGBMRegressor\t0.029332\t0.867618\n",
    "11\t12.LGBMRegressor\t0.029250\t0.867823\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "regrs = [\n",
    "    xgbt1, gbdt1, forest1, lgb1,\n",
    "    xgbt2, gbdt2, forest2, lgb2,\n",
    "    xgbt3, gbdt3, forest3, lgb3\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Stacker(object):\n",
    "    def __init__(self, n_splits, stacker, base_models):\n",
    "        self.n_splits = n_splits\n",
    "        self.stacker = stacker\n",
    "        self.base_models = base_models\n",
    "    \n",
    "    # X: 原始训练集, y: 原始训练集真实值, predict_data: 原始待预测数据\n",
    "    def fit_predict(self, X, y, predict_data):\n",
    "        X = np.array(X)\n",
    "        y = np.array(y)\n",
    "        T = np.array(predict_data)\n",
    "\n",
    "        folds = list(KFold(n_splits=self.n_splits, shuffle=False, random_state=2018).split(X, y))\n",
    "        \n",
    "        # 以基学习器预测结果为特征的 stacker的训练数据 与 stacker预测数据\n",
    "        S_train = np.zeros((X.shape[0], len(self.base_models)))\n",
    "        S_predict = np.zeros((T.shape[0], len(self.base_models)))\n",
    "        \n",
    "        for i, regr in enumerate(self.base_models):\n",
    "            print(i + 1, 'Base model:', str(regr).split('(')[0])\n",
    "            S_predict_i = np.zeros((T.shape[0], self.n_splits))\n",
    "            \n",
    "            for j, (train_idx, test_idx) in enumerate(folds):\n",
    "                # 将X分为训练集与测试集\n",
    "                X_train, y_train, X_test, y_test = X[train_idx], y[train_idx], X[test_idx], y[test_idx]\n",
    "                print ('Fit fold', (j+1), '...')\n",
    "                regr.fit(X_train, y_train)\n",
    "                y_pred = regr.predict(X_test)                \n",
    "                S_train[test_idx, i] = y_pred\n",
    "                S_predict_i[:, j] = regr.predict(T)\n",
    "            \n",
    "            S_predict[:, i] = S_predict_i.mean(axis=1)\n",
    "\n",
    "        nmse_score = cross_val_score(self.stacker, S_train, y, cv=5, scoring='neg_mean_squared_error')\n",
    "        print('CV MSE:', -nmse_score)\n",
    "        print('Stacker AVG MSE:', -nmse_score.mean(), 'Stacker AVG Score:', np.mean(np.divide(1, 1 + np.sqrt(-nmse_score))))\n",
    "\n",
    "        self.stacker.fit(S_train, y)\n",
    "        res = self.stacker.predict(S_predict)\n",
    "        return res, S_train, S_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "2 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "3 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "4 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "5 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "6 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "7 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "8 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "9 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "10 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "11 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "12 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "CV MSE: [0.05486121 0.01272405 0.0165533  0.01482696 0.01553867]\n",
      "Stacker AVG MSE: 0.02290083961944828 Stacker AVG Score: 0.8750957884585737\n"
     ]
    }
   ],
   "source": [
    "# stacking_mode1 = Ridge(alpha=0.008, copy_X=True, fit_intercept=False, solver='auto', random_state=2)\n",
    "stacking_model = SVR(C=100, gamma=0.01, epsilon=0.01)\n",
    "stacker = Stacker(5, stacking_model, regrs)\n",
    "pred_stack, S_train_data, S_predict_data = stacker.fit_predict(all_X_train, all_y_train, test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CV MSE: [0.00834157 0.01945687 0.0095695  0.08376848 0.01302208]\n",
    "Stacker AVG MSE: 0.02683170221309707 Stacker AVG Score: 0.8755806306487525"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "2 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "3 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "4 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "5 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "6 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "7 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "8 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "9 Base model: XGBRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "10 Base model: GradientBoostingRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "11 Base model: RandomForestRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "12 Base model: LGBMRegressor\n",
      "Fit fold 1 ...\n",
      "Fit fold 2 ...\n",
      "Fit fold 3 ...\n",
      "Fit fold 4 ...\n",
      "Fit fold 5 ...\n",
      "CV MSE: [0.00913498 0.01777301 0.05715313 0.01018346 0.01023765]\n",
      "Stacker AVG MSE: 0.02089644674370355 Stacker AVG Score: 0.8837278233083946\n"
     ]
    }
   ],
   "source": [
    "stacking_model2 = SVR(C=100, gamma=0.01, epsilon=0.01)\n",
    "stacker2 = Stacker(5, stacking_model2, regrs)\n",
    "pred_clean_stack, S_clean_train_data, S_clean_predict_data = stacker2.fit_predict(clean_X, clean_y, clean_sub_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CV MSE: [0.01659384 0.00626714 0.01261921 0.05345451 0.01616147]\n",
    "Stacker AVG MSE: 0.021019233812974467 Stacker AVG Score: 0.8821919685074249"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CV MSE: [0.00902831 0.01733493 0.05746076 0.01078939 0.00961003]\n",
    "Stacker AVG MSE: 0.020844682710295333 Stacker AVG Score: 0.8840294650706134"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = all_ID.copy()\n",
    "submission['Detection'] = pred_stack # !!!!!预测结果pred放在这个,其他的不用管\n",
    "submission=pd.merge(test_ID,submission,on='ID',how='left')\n",
    "submission=submission.fillna(3.799931e-01)\n",
    "submission.to_csv('000000000.csv',index=None,sep=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_result['score'] = pred_stack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "index = df_result[df_result['ID'].isin(special_missing_ID)].index\n",
    "df_result.loc[index, 'score'] = 0.379993053"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_index = df_result[df_result['ID'].isin(cleaned_sub_data_ID)].index\n",
    "df_result.loc[c_index, 'score'] = pred_clean_stack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result.to_csv('8_16_time_16.19.version4.0.csv', index=False, header=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
